from bs4 import BeautifulSoup
import requests
import time
import pymysql

db = pymysql.connect('localhost', 'root', 'root', 'happy')


def get_links_from(channel, pages, who_sells=0):
    '''

    :param channel:
    :param pages:
    :param who_sells:
    :return:
    '''
    # http://bj.58.com/diannao/0/pn1/
    list_url = '{}/{}/pn{}/'.format(channel, str(who_sells), str(pages))

    print(list_url)

    wb_data = requests.get(list_url)
    time.sleep(2)

    soup = BeautifulSoup(wb_data.text, 'lxml')

    link = soup.select('td.t > a.t')



    cursor = db.cursor()
    for link_item in link:
        link_url = link_item.get('href').split('?')[0]
        print(link_url)
        sql = "INSERT INTO h_list_link(list_link) VALUES ('%s')" % (link_url)
        try:
            # 执行sql语句
            cursor.execute(sql)
            # 提交到数据库执行
            db.commit()
        except:
            # 如果发生错误则回滚
            db.rollback()
    # 关闭数据库连接
    db.close()


get_links_from('http://bj.58.com/diannao', 1)
